package com.example.myFirstWebmagic.airQualityForecasting;

import java.util.ArrayList;
import java.util.List;

import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.example.myFirstWebmagic.pipeline.ConsolePipelineDemo;
import com.example.myFirstWebmagic.util.DateUtil;
import com.example.myFirstWebmagic.util.StringUtil;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonParser;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 
 * ClassName: ProvinceForecastProcess 
 * @Description: 省域空气质量形势预报
 * @author xiongyao
 * @date 2017年3月13日
 */
public class ProvinceForecastProcess implements PageProcessor{

	//http://106.37.208.228:8082/ProvinceForecast/GetGlobalInfo
	
	private Site site = Site.me().setRetryTimes(10).setSleepTime(5000);
	
	Logger log=LoggerFactory.getLogger(ProvinceForecastProcess.class);
	
	
	@Override
	public void process(Page page) {
		// TODO Auto-generated method stub
		Gson gson=new Gson();
		JsonArray array=new JsonParser().parse(page.getJson().toString()).getAsJsonArray();
		List<EnProvinceForecast> forecasts=new ArrayList<EnProvinceForecast>();
		for(JsonElement element:array){
			EnProvinceForecast forecast=gson.fromJson(element,EnProvinceForecast.class);
			ProvinceForecastPK id=new ProvinceForecastPK();
			id.setName(forecast.getProvinceName());
			id.setDate(DateUtil.getToday());
			forecast.setId(id);
			forecasts.add(forecast);
		}
		page.putField("objectList", forecasts);
	}

	@Override
	public Site getSite() {
		// TODO Auto-generated method stub
		return this.site;
	}
	
	@Test
	public void test(){
		// TODO Auto-generated method stub
		String [] startUrl=new String[]{"http://106.37.208.228:8082/ProvinceForecast/GetGlobalInfo"};
		// TODO Auto-generated method stub
		Spider.create(new ProvinceForecastProcess())
        //从"http://datacenter.mep.gov.cn/index!getCityMain.action"开始抓
        .addUrl(startUrl)
        .addPipeline(new ConsolePipelineDemo())
        //开启5个线程抓取
        .thread(5)
        //启动爬虫
        .run();
	}
	
}
